{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%run -n main.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path = join_path(DATA_DIR, DATASET)\n",
    "# !mkdir -p {path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for name in [NE5, BSNLP, FACTRU, GAREEV]:\n",
    "#     path = join_path(CORUS_DATA_DIR, CORUS_FILES[name])\n",
    "#     records = LOADS[name](path)\n",
    "#     records = (_.adapted for _ in records)\n",
    "#     records = log_progress(records, desc=name)\n",
    "\n",
    "#     path = join_path(DATA_DIR, DATASET, name + JL + GZ)\n",
    "#     items = as_jsons(records)\n",
    "#     lines = format_jl(items)\n",
    "#     dump_gz_lines(lines, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -n main.py\n",
    "datasets = {}\n",
    "for name in [NE5, BSNLP, FACTRU, GAREEV]:\n",
    "    path = join_path(DATA_DIR, DATASET, name + JL + GZ)\n",
    "    lines = load_gz_lines(path)\n",
    "    items = parse_jl(lines)\n",
    "    datasets[name] = list(from_jsons(items, Markup))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for name in MODELS:\n",
    "#     path = join_path(DATA_DIR, name)\n",
    "#     !mkdir -p {path}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# docker = docker_client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_name = SPACY  # MITIE, TOMITA, TEXTERRA, SLOVNET\n",
    "# model = MODELS[model_name]()\n",
    "# model.start(docker)\n",
    "# model.wait()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for dataset_name in [NE5, BSNLP, FACTRU, GAREEV]:\n",
    "#     records = model.map(_.text for _ in datasets[dataset_name])\n",
    "#     records = (_.adapted for _ in records)\n",
    "#     records = log_progress(records, desc=dataset_name)\n",
    "\n",
    "#     path = join_path(DATA_DIR, model_name, dataset_name + JL + GZ)\n",
    "#     items = as_jsons(records)\n",
    "#     lines = format_jl(items)\n",
    "#     dump_gz_lines(lines, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.stop(docker)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # select cuda 10.0, high bandwith\n",
    "# !vast search offers | grep '1 x  GTX 1080 Ti'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %run -n main\n",
    "# model = DeeppavlovModel()\n",
    "# model = DeeppavlovBERTModel()\n",
    "# model = SlovnetBERTModel()\n",
    "# model = StanzaModel()\n",
    "# model = DeeppavlovSlavicModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !vast create instance 577392 --image {model.image} --disk 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !vast show instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ssh ssh5.vast.ai -p 19600 -l root -Nf -L {model.port}:localhost:{model.container_port}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for dataset_name in [NE5, BSNLP, FACTRU, GAREEV]:\n",
    "#     records = datasets[dataset_name]\n",
    "#     records = log_progress(records, desc=dataset_name)\n",
    "#     records = model.map(_.text for _ in records)\n",
    "#     records = (_.adapted for _ in records)\n",
    "\n",
    "#     path = join_path(DATA_DIR, model.name, dataset_name + JL + GZ)\n",
    "#     items = as_jsons(records)\n",
    "#     lines = format_jl(items)\n",
    "#     dump_gz_lines(lines, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !vast destroy instance 585391"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_models = {}\n",
    "for dataset in DATASETS:\n",
    "    for model in MODELS:\n",
    "        path = join_path(DATA_DIR, model, dataset + JL + GZ)\n",
    "        lines = load_gz_lines(path)\n",
    "        items = parse_jl(lines)\n",
    "        dataset_models[dataset, model] = list(from_jsons(items, Markup))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = {}\n",
    "for dataset, model in log_progress(dataset_models):\n",
    "    preds = dataset_models[dataset, model]\n",
    "    targets = datasets[dataset]\n",
    "    scores[dataset, model] = score_markups(preds, targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_table = scores_report_table(scores, DATASETS, MODELS)\n",
    "html = format_scores_report(scores_table)\n",
    "HTML(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = format_github_scores_report(scores_table)\n",
    "patch_readme(NER1, html, README)\n",
    "patch_readme(NER1, html, SLOVNET_README)\n",
    "HTML(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BENCH = [\n",
    "    # GTX 1080 Ti\n",
    "    Bench(\n",
    "        DEEPPAVLOV,\n",
    "        init=5.9,\n",
    "        disk=1 * GB,  # 1GB emb + 5MB model\n",
    "        ram=3 * GB,\n",
    "        speed=24.31,  # 1.95 / 7 cores on CPU,\n",
    "        device=GPU\n",
    "    ),\n",
    "    Bench(\n",
    "        DEEPPAVLOV_BERT,\n",
    "        init=34.5,\n",
    "        disk=2 * GB,\n",
    "        ram=6 * GB,\n",
    "        speed=13.13,  # 17.71 / 3 cores on CPU\n",
    "        device=GPU\n",
    "    ),\n",
    "    Bench(\n",
    "        DEEPPAVLOV_SLAVIC,\n",
    "        init=35,\n",
    "        disk=2 * GB,\n",
    "        ram=4 * GB,\n",
    "        speed=8,\n",
    "        device=GPU\n",
    "    ),\n",
    "    Bench(\n",
    "        SLOVNET_BERT,\n",
    "        init=5,\n",
    "        disk=473 * MB,\n",
    "        ram=9500 * MB,\n",
    "        speed=40,\n",
    "        device=GPU\n",
    "    ),\n",
    "    Bench(\n",
    "        SLOVNET,\n",
    "        init=1,\n",
    "        disk=27 * MB,\n",
    "        ram=205 * MB,\n",
    "        speed=25.3,\n",
    "    ),\n",
    "    \n",
    "    # 16 CPUs\n",
    "    Bench(\n",
    "        PULLENTI,\n",
    "        init=2.85,\n",
    "        disk=16 * MB,\n",
    "        ram=253 * MB,\n",
    "        speed=6.05\n",
    "    ),\n",
    "    Bench(\n",
    "        TEXTERRA,\n",
    "        init=47.6,\n",
    "        disk=193 * MB,\n",
    "        ram=3.3 * GB,  # leaks\n",
    "        speed=20.16 / 5  # utils ~5 cores\n",
    "    ),\n",
    "    Bench(\n",
    "        TOMITA,\n",
    "        init=2.03,\n",
    "        disk=64 * MB,\n",
    "        ram=63 * MB,\n",
    "        speed=29.8,\n",
    "    ),\n",
    "    Bench(\n",
    "        MITIE,\n",
    "        init=28.3,\n",
    "        disk=327 * MB,\n",
    "        ram=261 * MB,\n",
    "        speed=32.8,\n",
    "    ), \n",
    "    Bench(\n",
    "        SPACY,\n",
    "        init=8,\n",
    "        disk=140 * MB,\n",
    "        ram=625 * MB,\n",
    "        speed=8,\n",
    "    ),\n",
    "    Bench(\n",
    "        STANZA,\n",
    "        init=3,\n",
    "        disk=591 * MB,\n",
    "        ram=11 * GB,\n",
    "        speed=3,\n",
    "        device=GPU\n",
    "    ),\n",
    "]\n",
    "\n",
    "bench_table = bench_report_table(BENCH, MODELS)\n",
    "html = format_bench_report(bench_table)\n",
    "patch_readme(NER2, html, README)\n",
    "patch_readme(NER2, html, SLOVNET_README)\n",
    "HTML(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
